/*
 * Hyuga: A Density-Grid Stream Clustering Platform.
 * Copyright (C) 2014 PUC-Rio/Laboratory for Advanced Collaboration
 *
 * Hyuga is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Hyuga is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Hyuga.  If not, see <http://www.gnu.org/licenses/>.
 */
package br.pucrio.inf.lac.konoha.hyuga.others.csv;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TreeMap;

import br.pucrio.inf.lac.konoha.hyuga.core.Output;
import br.pucrio.inf.lac.konoha.hyuga.core.output.CSVEntry;
import br.pucrio.inf.lac.konoha.hyuga.core.output.OutputType;
import static br.pucrio.inf.lac.konoha.hyuga.others.csv.ClusteringStat.*;

public class CSVEnrichedRandIndex {
    File dg2cepFile;
    File dbscanFile;
    PrintWriter randIndexFW;
    
    Output outSingleton;
    ClusteringMeasurementTool measurementTool;
    
    private TreeMap<Long, List<CSVEntry>> dg2cepSecBySecEntries;
    private TreeMap<Long, List<CSVEntry>> dbscanSecBySecEntries;
    
    public static final double randLowThreshold  = 0.5;
    public static final double randCompThreshold = 0.5;
        
    public CSVEnrichedRandIndex(String dg2cep, String dbscan, String output) {
        dg2cepFile = new File(dg2cep);
        dbscanFile = new File(dbscan);
        outSingleton = Output.getInstance();
        measurementTool = ClusteringMeasurementTool.getInstance();
        
        if (!dg2cepFile.exists()) {
            System.out.println("File " + dg2cepFile.getName() + " does not exists.");
            System.exit(-1);
        }

        if (!dbscanFile.exists()) {
            System.out.println("File " + dbscanFile.getName() + " does not exists.");
            System.exit(-1);
        }

        try {
            File randIndexFile = new File(output + "randIndex.csv");
            randIndexFile.createNewFile();
            randIndexFW = new PrintWriter(new BufferedWriter(new FileWriter(randIndexFile)));
        } catch (IOException ex) {
            System.out.println("Could not create output files");
            System.exit(-1);
        }
    }
    
    public CSVEntry getDG2CEPEntry(String csvLine) {
        // FirstDivision [0] timestamp,type,[cell],size
        // FirstDivision [1] cellcontent
        String[] firstDivision = new String(csvLine).split("\"");
        String[] entrydata = new String(firstDivision[0]).split(",");
        
        long timestamp = Long.parseLong(new String(entrydata[0]));
        OutputType type = OutputType.valueOf(new String(entrydata[1]));
        
        if (type != OutputType.NONE) {
            int size = Integer.parseInt(new String(entrydata[3]));
            String[] cellContent   = new String(firstDivision[1]).split(",");
            Set<String> contentSet = new HashSet<String>(Arrays.asList(cellContent));
            
            return new CSVEntry(timestamp, type, size, null, contentSet);
        } else {
            return new CSVEntry(timestamp, type);
        }
    }
    
    public CSVEntry getDBSCANEntry(String csvLine) {
        // FirstDivision [0] timestamp,snapshotsize,totaltime,size
        // FirstDivision [1] cellcontent
        String[] firstDivision = new String(csvLine).split("\"");
        String[] entrydata = new String(firstDivision[0]).split(",");
        
        long timestamp = Long.parseLong(new String(entrydata[0]));
        OutputType type = OutputType.DBSCAN;
        int size = Integer.parseInt(new String(entrydata[3]));

        String[] cellContent   = new String(firstDivision[1]).split(",");
        Set<String> contentSet = new HashSet<String>(Arrays.asList(cellContent));
                
        return new CSVEntry(timestamp, type, size, null, contentSet);
    }

    private TreeMap<Long,List<CSVEntry>> dg2cepSecBySecParser(BufferedReader dg2cepBuffer) {
        TreeMap<Long,List<CSVEntry>> dg2cepSecBySecEntries = new TreeMap<Long, List<CSVEntry>>();
        String dg2cepCSVLine = "";
        int lineCount = 0;
        
        // Declare only once the variables
        CSVEntry dg2cepEntry = null;
        long entryTimestamp  = 0;
        List<CSVEntry> clusterListInThisSecond = null;
        
        try {
            dg2cepBuffer.readLine(); // header
            while((dg2cepCSVLine = dg2cepBuffer.readLine()) != null) {
                System.out.println("DG2CEP linha " + ++lineCount);
                
                dg2cepEntry = getDG2CEPEntry(dg2cepCSVLine);
                entryTimestamp = dg2cepEntry.getTimestamp();

                if (!dg2cepSecBySecEntries.containsKey(entryTimestamp)) {
                    dg2cepSecBySecEntries.put(entryTimestamp, new ArrayList<CSVEntry>());
                }
                
                clusterListInThisSecond = dg2cepSecBySecEntries.get(entryTimestamp);
                
                if (dg2cepEntry.getType() == OutputType.NONE) { 
                    continue;
                } else { 
                    clusterListInThisSecond.add(dg2cepEntry);
                }
            }
        } catch (IOException e) {
            System.out.println("Exception at reading dg2cep output file");
        }
        
        return dg2cepSecBySecEntries;
    } 
    
   private TreeMap<Long,List<CSVEntry>> dbscanSecBySecParser(BufferedReader dbscanBuffer) {
       TreeMap<Long,List<CSVEntry>> dbscanSecBySecEntries = new TreeMap<Long, List<CSVEntry>>();
       String dbscanCSVLine = "";
       int lineCount = 0;

       // Declare only once the variables
       CSVEntry dbscanEntry = null;
       long entryTimestamp  = 0;
       List<CSVEntry> clusterListInThisSecond = null;

       try {
           dbscanBuffer.readLine(); // header
           while((dbscanCSVLine = dbscanBuffer.readLine()) != null) {
               System.out.println("DBSCAN linha " + ++lineCount);
               
               dbscanEntry = getDBSCANEntry(dbscanCSVLine);
               entryTimestamp = dbscanEntry.getTimestamp();

               if (!dbscanSecBySecEntries.containsKey(entryTimestamp)) {
                   dbscanSecBySecEntries.put(entryTimestamp, new ArrayList<CSVEntry>());
               }
               
               clusterListInThisSecond = dbscanSecBySecEntries.get(entryTimestamp);
               clusterListInThisSecond.add(dbscanEntry);
           }
       } catch (IOException e) {
           System.out.println("Exception at reading dbscan output file");
       }
       
       return dbscanSecBySecEntries;
   }
    
    private void calculateAllRandIndex() throws IOException {
        BufferedReader dg2cepBuffer = new BufferedReader(new FileReader(dg2cepFile));
        BufferedReader dbscanBuffer = new BufferedReader(new FileReader(dbscanFile));
        
        // Parsing the CSV files
        dg2cepSecBySecEntries = dg2cepSecBySecParser(dg2cepBuffer);
        System.gc();
        dbscanSecBySecEntries = dbscanSecBySecParser(dbscanBuffer);
        dg2cepBuffer.close();
        dbscanBuffer.close();
        
        // Totals (Number)
        int numberOfClustersEntries = 0;
        int numberOfDetectedClusters = 0;
        int numberOfDBSCANClusters = 0;
        int numberOfDG2CEPClusters = 0;
        
        // Totals
        List<Double> totalAdjustedRandIndexList = new ArrayList<>();
        List<Double> totalRandIndexList = new ArrayList<>();
        List<Double> totalRoundFalsePositive = new ArrayList<>();
        List<Double> totalRoundFalseNegative = new ArrayList<>();
        List<Double> totalFalsePositive = new ArrayList<>();
        List<Double> totalFalseNegative = new ArrayList<>();
        
        List<Double> totalDetectedFullRandIndex = new ArrayList<>();
        List<Double> totalDetectedRandIndex = new ArrayList<>();
        List<Double> totalDetectedJaccardIndex = new ArrayList<>();
        List<Double> totalDetectedPrecision = new ArrayList<>();
        List<Double> totalDetectedRecall = new ArrayList<>();
        List<Double> totalDetectedFMeasure = new ArrayList<>();
        
        long dbscanMinimumTimestamp = dbscanSecBySecEntries.firstKey();
        long dbscanMaximumTimestamp = dbscanSecBySecEntries.lastKey();
        long entryTimestamp = dbscanMinimumTimestamp;

        randIndexFW.println("second,gadjri,gri,fp,fn,detfullri,detri,detjaccard,detprecision,detrecall,detfmeasure");

        // Iterating
        while (entryTimestamp <= dbscanMaximumTimestamp) {
            List<CSVEntry> dg2cepEntries = dg2cepSecBySecEntries.get(entryTimestamp);
            List<CSVEntry> dbscanEntries = dbscanSecBySecEntries.get(entryTimestamp);

            int dg2cepFound = 0;
            int dbscanFound = 0;
            int numberClusterFound = 0;

            // Round (to be put in totals)
            double roundAdjRandIndex = 0;
            double roundRandIndex = 0;
            
            double roundFalsePositive = 0;
            double roundFalseNegative = 0;
            
            double roundDetectedFullRandIndex = 0;
            double roundDetectedRandIndex = 0;
            double roundDetectedJaccardIndex = 0;
            double roundDetectedPrecision = 0;
            double roundDetectedRecall = 0;
            double roundDetectedFMeasure = 0;
            
            numberOfClustersEntries++;
            Set<CSVEntry> detectedClusters = new HashSet<>();

            if (dg2cepEntries != null && dg2cepEntries.size() > 0) {
                dg2cepFound = dg2cepEntries.size();
                numberOfDG2CEPClusters += dg2cepFound;                
                
                if (dbscanEntries == null) {
                    roundFalsePositive = dg2cepFound;
                } else {
                    // Global
                    roundAdjRandIndex = measurementTool.adjustedRandIndex(dg2cepEntries, dbscanEntries);
                    roundRandIndex = measurementTool.adjustedRandIndex(dg2cepEntries, dbscanEntries, false);
                    
                    totalAdjustedRandIndexList.add(roundAdjRandIndex);
                    totalRandIndexList.add(roundRandIndex);
                    
                    dbscanFound = dbscanEntries.size();
                    numberOfDBSCANClusters += dbscanFound;
                    
                    // Detected
                    for (CSVEntry dg2cepEntry : dg2cepEntries) {
                        CSVEntry dbscanEntry = measurementTool.getEntryWithHigherRandIndex(dg2cepEntry, dbscanEntries);
                        double fullRandIndex = measurementTool.realRandIndex(dg2cepEntry, dbscanEntry);
                        double randIndex     = measurementTool.randIndex(dg2cepEntry, dbscanEntry);
                        double jaccardIndex  = measurementTool.jaccardDistance(dg2cepEntry, dbscanEntry);
                        double precision     = measurementTool.precision(dg2cepEntry, dbscanEntry);
                        double recall        = measurementTool.recall(dg2cepEntry, dbscanEntry);
                        double fMeasure      = measurementTool.fMeasure(dg2cepEntry, dbscanEntry);

                        if (recall > randLowThreshold) {
                            detectedClusters.add(dbscanEntry);
                            numberClusterFound++;
                            numberOfDetectedClusters++;
                            
                            roundDetectedFullRandIndex += fullRandIndex;
                            roundDetectedRandIndex     += randIndex;
                            roundDetectedJaccardIndex  += jaccardIndex;
                            roundDetectedPrecision     += precision;
                            roundDetectedRecall        += recall;
                            roundDetectedFMeasure      += fMeasure;
                            
                            totalDetectedFullRandIndex.add(fullRandIndex);
                            totalDetectedRandIndex.add(randIndex);
                            totalDetectedJaccardIndex.add(jaccardIndex);
                            totalDetectedPrecision.add(precision);
                            totalDetectedRecall.add(recall);
                            totalDetectedFMeasure.add(fMeasure);
                        }
                    }
                }

                if (numberClusterFound > 0) {
                    roundDetectedFullRandIndex = roundDetectedFullRandIndex / numberClusterFound;
                    roundDetectedRandIndex     = roundDetectedRandIndex / numberClusterFound;
                    roundDetectedJaccardIndex  = roundDetectedJaccardIndex / numberClusterFound;
                    roundDetectedPrecision     = roundDetectedPrecision / numberClusterFound;
                    roundDetectedRecall        = roundDetectedRecall / numberClusterFound;
                    roundDetectedFMeasure      = roundDetectedFMeasure / numberClusterFound;
                } else {
                    roundDetectedFullRandIndex = 0;
                    roundDetectedRandIndex     = 0;
                    roundDetectedJaccardIndex  = 0;
                    roundDetectedPrecision     = 0;
                    roundDetectedRecall        = 0;
                    roundDetectedFMeasure      = 0;
                }

                // False Positive and Negative
                roundFalsePositive = dg2cepFound - numberClusterFound;
                roundFalseNegative = dbscanFound - detectedClusters.size();
                
                totalFalsePositive.add(roundFalsePositive);
                totalFalseNegative.add(roundFalseNegative);
                
                roundFalsePositive = roundFalsePositive / dg2cepFound;
                roundFalseNegative = roundFalseNegative / dbscanFound;

                totalRoundFalsePositive.add(roundFalsePositive);
                totalRoundFalseNegative.add(roundFalseNegative);
            } else {
                if (dbscanEntries == null) { // Both (DG2CEP) and (DBSCAN) are null
                    numberOfDetectedClusters += 1;
                    
                    // Global
                    roundAdjRandIndex = 1.0;
                    roundRandIndex = 1.0;
                    totalAdjustedRandIndexList.add(roundAdjRandIndex);
                    totalRandIndexList.add(roundRandIndex);
                    
                    // Detected
                    roundDetectedFullRandIndex = 1.0;
                    roundDetectedRandIndex     = 1.0;
                    roundDetectedJaccardIndex  = 1.0;
                    roundDetectedPrecision     = 1.0;
                    roundDetectedRecall        = 1.0;
                    roundDetectedFMeasure      = 1.0;

                    totalDetectedFullRandIndex.add(1.0);
                    totalDetectedRandIndex.add(1.0);
                    totalDetectedJaccardIndex.add(1.0);
                    totalDetectedPrecision.add(1.0);
                    totalDetectedRecall.add(1.0);
                    totalDetectedFMeasure.add(1.0);
                    
                    // False Positive and Negative (no change, since it starts as 0)
                } else {
                    numberOfDBSCANClusters += dbscanEntries.size();
                    
                    roundFalseNegative = dbscanEntries.size() - numberClusterFound;
                    totalFalseNegative.add(roundFalseNegative);

                    totalRoundFalseNegative.add(1.0);
                }
            }

            DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
            Calendar cal = Calendar.getInstance();
            int seconds = (int) (entryTimestamp  / 1000) % 60 ;
            int minutes = (int) ((entryTimestamp / (1000*60)) % 60) - 5;
            int hours   = (int) ((entryTimestamp / (1000*60*60)) % 24);
            hours+= 17;

            cal.set(Calendar.HOUR_OF_DAY, hours);
            cal.set(Calendar.MINUTE, minutes);
            cal.set(Calendar.SECOND, seconds);
            cal.set(Calendar.DAY_OF_MONTH, 24);
            cal.set(Calendar.MONTH, Calendar.JULY);
            cal.set(Calendar.YEAR, 2014);
            
            System.out.println(dateFormat.format(cal.getTime()) + "," + roundAdjRandIndex + "," + roundRandIndex + ","
                             + roundFalsePositive + "," + roundFalseNegative + ","
                             + roundDetectedFullRandIndex + "," + roundDetectedRandIndex + "," + roundDetectedJaccardIndex + ","
                             + roundDetectedPrecision + "," + roundDetectedRecall + "," + roundDetectedFMeasure);
            randIndexFW.println(dateFormat.format(cal.getTime()) + "," + roundAdjRandIndex + "," + roundRandIndex + ","
                              + roundFalsePositive + "," + roundFalseNegative + ","
                              + roundDetectedFullRandIndex + "," + roundDetectedRandIndex + "," + roundDetectedJaccardIndex + ","
                              + roundDetectedPrecision + "," + roundDetectedRecall + "," + roundDetectedFMeasure);

            // Incrementing timestamp
            entryTimestamp += 1000;
        }
        
        System.out.println("(GLOBAL)    ADJ RAND INDEX: " + (sum(totalAdjustedRandIndexList) / numberOfClustersEntries) + " \t " + average(totalAdjustedRandIndexList) + " \t " + standardDeviation(totalAdjustedRandIndexList) + "\t" + numberOfClustersEntries);
        System.out.println("(GLOBAL)        RAND INDEX: " + (sum(totalRandIndexList) / numberOfClustersEntries) + " \t " + average(totalRandIndexList) + " \t " + standardDeviation(totalRandIndexList) + "\t" + numberOfClustersEntries);
        System.out.println("(GLOBAL)    False Positive: " + (sum(totalRoundFalsePositive) / numberOfClustersEntries) + "\t " + average(totalRoundFalsePositive) + " \t " + standardDeviation(totalRoundFalsePositive) + "\t" + numberOfClustersEntries);
        System.out.println("(GLOBAL)    False Negative: " + (sum(totalRoundFalseNegative) / numberOfClustersEntries) + "\t " + average(totalRoundFalseNegative) + " \t " + standardDeviation(totalRoundFalseNegative) + "\t" + numberOfClustersEntries);
        System.out.println("(BY DG2CEP) False Positive: " + (sum(totalFalsePositive) / numberOfDG2CEPClusters));
        System.out.println("(BY DBSCAN) False Negative: " + (sum(totalFalseNegative) / numberOfDBSCANClusters));
        System.out.println();
        System.out.println("(DETECTED) FULL RAND INDEX: " + (sum(totalDetectedFullRandIndex) / numberOfDetectedClusters) + " \t " + average(totalDetectedFullRandIndex) + " \t " + standardDeviation(totalDetectedFullRandIndex) + "\t" + numberOfDetectedClusters);
        System.out.println("(DETECTED)      RAND INDEX: " + (sum(totalDetectedRandIndex) / numberOfDetectedClusters) + " \t " + average(totalDetectedRandIndex) + " \t " + standardDeviation(totalDetectedRandIndex) + "\t" + numberOfDetectedClusters);
        System.out.println("(DETECTED)   JACCARD INDEX: " + (sum(totalDetectedJaccardIndex) / numberOfDetectedClusters) + " \t " + average(totalDetectedJaccardIndex) + " \t " + standardDeviation(totalDetectedJaccardIndex) + "\t" + numberOfDetectedClusters); 
        System.out.println("(DETECTED)       PRECISION: " + (sum(totalDetectedPrecision) / numberOfDetectedClusters) + " \t " + average(totalDetectedPrecision) + " \t " + standardDeviation(totalDetectedPrecision) + "\t" + numberOfDetectedClusters);
        System.out.println("(DETECTED)          RECALL: " + (sum(totalDetectedRecall) / numberOfDetectedClusters) + " \t " + average(totalDetectedRecall) + " \t " + standardDeviation(totalDetectedRecall) + "\t" + numberOfDetectedClusters);
        System.out.println("(DETECTED)       F-MEASURE: " + (sum(totalDetectedFMeasure) / numberOfDetectedClusters) + " \t " + average(totalDetectedFMeasure) + " \t " + standardDeviation(totalDetectedFMeasure) + "\t" + numberOfDetectedClusters);


        randIndexFW.flush();
        randIndexFW.close();
    }
    
    /**
     * Main Routine.
     * 
     * @param args a CSV filename.
     * @throws IOException If log files cannot be created.
     * @throws IllegalAccessException if args is null. 
     */
    public static void main(String[] args) throws IOException {
        if (args.length != 3) {
            System.out.println("USAGE: java -jar CSVRandIndex dg2cepFile.csv dbscanFile.csv outputFile.csv");
        }
        
        CSVEnrichedRandIndex csvtool = new CSVEnrichedRandIndex(args[0], args[1], args[2]);
        csvtool.calculateAllRandIndex();
    }
}

